{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import random\n",
    "\n",
    "import pandas as pd\n",
    "import jieba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content_id</th>\n",
       "      <th>content</th>\n",
       "      <th>subject</th>\n",
       "      <th>sentiment_value</th>\n",
       "      <th>sentiment_word</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>vUXizsqexyZVRdFH</td>\n",
       "      <td>因为森林人即将换代，这套系统没必要装在一款即将换代的车型上，因为肯定会影响价格。</td>\n",
       "      <td>价格</td>\n",
       "      <td>0</td>\n",
       "      <td>影响</td>\n",
       "      <td>价格::0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4QroPd9hNfnCHVt7</td>\n",
       "      <td>四驱价格貌似挺高的，高的可以看齐XC60了，看实车前脸有点违和感。不过大众的车应该不会差。</td>\n",
       "      <td>价格</td>\n",
       "      <td>-1</td>\n",
       "      <td>高</td>\n",
       "      <td>价格::-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>QmqJ2AvM5GplaRyz</td>\n",
       "      <td>斯柯达要说质量，似乎比大众要好一点，价格也低一些，用料完全一样。我听说过野帝，但没听说过你说...</td>\n",
       "      <td>价格</td>\n",
       "      <td>1</td>\n",
       "      <td>低</td>\n",
       "      <td>价格::1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>KMT1gFJiU4NWrVDn</td>\n",
       "      <td>这玩意都是给有钱任性又不懂车的土豪用的，这价格换一次我妹夫EP020可以换三锅了</td>\n",
       "      <td>价格</td>\n",
       "      <td>-1</td>\n",
       "      <td>有钱任性</td>\n",
       "      <td>价格::-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>nVIlGd5yMmc37t1o</td>\n",
       "      <td>17价格忒高，估计也就是14-15左右。</td>\n",
       "      <td>价格</td>\n",
       "      <td>-1</td>\n",
       "      <td>高</td>\n",
       "      <td>价格::-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         content_id                                            content  \\\n",
       "0  vUXizsqexyZVRdFH           因为森林人即将换代，这套系统没必要装在一款即将换代的车型上，因为肯定会影响价格。   \n",
       "1  4QroPd9hNfnCHVt7      四驱价格貌似挺高的，高的可以看齐XC60了，看实车前脸有点违和感。不过大众的车应该不会差。   \n",
       "2  QmqJ2AvM5GplaRyz  斯柯达要说质量，似乎比大众要好一点，价格也低一些，用料完全一样。我听说过野帝，但没听说过你说...   \n",
       "3  KMT1gFJiU4NWrVDn           这玩意都是给有钱任性又不懂车的土豪用的，这价格换一次我妹夫EP020可以换三锅了   \n",
       "4  nVIlGd5yMmc37t1o                            17价格忒高，估计也就是14-15左右。      \n",
       "\n",
       "  subject  sentiment_value sentiment_word   label  \n",
       "0      价格                0             影响   价格::0  \n",
       "1      价格               -1              高  价格::-1  \n",
       "2      价格                1              低   价格::1  \n",
       "3      价格               -1           有钱任性  价格::-1  \n",
       "4      价格               -1              高  价格::-1  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = \"data/origin/train.csv\"\n",
    "df = pd.read_csv(path)\n",
    "df[\"label\"] = df[\"subject\"] + \"::\" +  df[\"sentiment_value\"].map(str)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "动力::0      1970\n",
       "价格::0      1014\n",
       "油耗::0       793\n",
       "操控::0       606\n",
       "配置::0       579\n",
       "舒适性::0      564\n",
       "动力::1       384\n",
       "安全性::0      380\n",
       "动力::-1      378\n",
       "操控::1       306\n",
       "内饰::0       271\n",
       "外观::0       263\n",
       "舒适性::-1     256\n",
       "空间::0       221\n",
       "配置::-1      154\n",
       "空间::1       154\n",
       "油耗::1       151\n",
       "内饰::-1      150\n",
       "价格::-1      145\n",
       "油耗::-1      138\n",
       "操控::-1      124\n",
       "配置::1       120\n",
       "外观::1       115\n",
       "内饰::1       115\n",
       "价格::1       114\n",
       "舒适性::1      111\n",
       "外观::-1      111\n",
       "安全性::1      100\n",
       "安全性::-1      93\n",
       "空间::-1       67\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"label\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 先保留多标签的数据\n",
    "mul_data = []\n",
    "single_data = []\n",
    "for c_id, gp in df.groupby(\"content_id\"):\n",
    "    if len(gp[\"subject\"]) > 1:\n",
    "        mul_data.append(gp)\n",
    "    else:\n",
    "        single_data.append(gp)\n",
    "\n",
    "df_mul = pd.concat(mul_data, axis=0)\n",
    "df_single = pd.concat(single_data, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 平衡1，-1，0实例的样本数量\n",
    "data = []\n",
    "for sub, gp in df_single.groupby(\"subject\"):\n",
    "    gp_mul = df_mul.query(\"subject==@sub\")\n",
    "    cnt_mul = gp_mul[\"sentiment_value\"].value_counts().to_dict()\n",
    "    cnt_single = gp[\"sentiment_value\"].value_counts().to_dict()\n",
    "    \n",
    "    z_num = int((cnt_mul[-1] + cnt_mul[1] + cnt_single[1] + cnt_single[-1]) * 1.5 // 2 - cnt_mul[0])\n",
    "    \n",
    "    pn_sample = gp.query(\"sentiment_value!=0\")\n",
    "    z_sample = gp.query(\"sentiment_value==0\").sample(min(z_num, cnt_single[0]))\n",
    "    data.append(pn_sample)\n",
    "    data.append(z_sample)\n",
    "    \n",
    "#     print(cnt_mul)\n",
    "#     print(cnt_single)\n",
    "#     print(z_num)\n",
    "#     break\n",
    "data.append(df_mul)\n",
    "\n",
    "df_last = pd.concat(data, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "动力::0      571\n",
       "动力::1      384\n",
       "动力::-1     378\n",
       "操控::0      322\n",
       "操控::1      306\n",
       "舒适性::0     275\n",
       "舒适性::-1    256\n",
       "油耗::0      216\n",
       "配置::0      205\n",
       "内饰::0      198\n",
       "价格::0      194\n",
       "外观::0      169\n",
       "空间::0      165\n",
       "配置::-1     154\n",
       "空间::1      154\n",
       "油耗::1      151\n",
       "内饰::-1     150\n",
       "价格::-1     145\n",
       "安全性::0     144\n",
       "油耗::-1     138\n",
       "操控::-1     124\n",
       "配置::1      120\n",
       "外观::1      115\n",
       "内饰::1      115\n",
       "价格::1      114\n",
       "舒适性::1     111\n",
       "外观::-1     111\n",
       "安全性::1     100\n",
       "安全性::-1     93\n",
       "空间::-1      67\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_last[\"label\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "traindata_path = \"data/car.train\"\n",
    "model_path = \"models/clf\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache /tmp/jieba.cache\n",
      "Loading model cost 0.646 seconds.\n",
      "Prefix dict has been built succesfully.\n"
     ]
    }
   ],
   "source": [
    "with open(traindata_path, 'w') as f:\n",
    "    for c_id, gp in df_last.groupby(\"content_id\"):\n",
    "        cont_ = gp['content'].values[0]\n",
    "        cont = \" \".join(jieba.lcut(cont_))\n",
    "        cont = re.sub(\"\\s[，。?？!！、\\*~“”…;；（）《》‘’]\", \"\", cont).lower()\n",
    "        label = gp[\"label\"].tolist()\n",
    "        line = \" \".join([\"__label__%s\"%l for l in label]) + \" \" + cont\n",
    "        f.write(line + \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'traindata_path' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-49ffdc052e94>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# 用fastText\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mfastText\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mFastText\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mclf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFastText\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_supervised\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtraindata_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwordNgrams\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'traindata_path' is not defined"
     ]
    }
   ],
   "source": [
    "# # 用fastText \n",
    "# from fastText import FastText\n",
    "# clf = FastText.train_supervised(input=traindata_path, wordNgrams=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'fasttext'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-c5c00c7017fe>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mfasttext\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'fasttext'"
     ]
    }
   ],
   "source": [
    "import fasttext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'fasttext' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-89654f8a677f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mclf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfasttext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msupervised\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_file\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtraindata_path\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0mwordNgrams\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.9\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m30\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m30\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mword_ngrams\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'fasttext' is not defined"
     ]
    }
   ],
   "source": [
    "clf = fasttext.supervised(input_file=traindata_path,   lr=0.9, epoch=30, dim=30, word_ngrams=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = \"多少有点吧，抓地力肯定提高了，但是动力损耗了，我觉得得不偿失。看你的需求是啥了。\"\n",
    "s= \"现在大家有了孩子都买SUV车型，确实，后备箱，空间很大，利于平时家庭使用。\"\n",
    "#s=\"刹车事关安全，还是去找师傅搞好过，省这钱真没必要。\"\n",
    "s=\"弄过2.5噪音大也颠，现在四轮都2.3还可以\"\n",
    "s = \"优惠多少钱亲？导航应该是在国内港口加装的\"\n",
    "s = \"你这热车太久了，会造成发动机积碳的加重，一般在热车30秒即可然后在行车中热车\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache /tmp/jieba.cache\n",
      "Loading model cost 0.610 seconds.\n",
      "Prefix dict has been built succesfully.\n"
     ]
    }
   ],
   "source": [
    "s_ = \" \".join(jieba.lcut(s))\n",
    "s_new = re.sub(\"\\s[，。?？!！、\\*~“”…;；（）《》‘’]\", \"\", s_).lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'你 这 热车 太久 了 会 造成 发动机 积碳 的 加重 一般 在 热车 30 秒 即可 然后 在 行车 中热车'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([['__label__动力::0',\n",
       "   '__label__动力::-1',\n",
       "   '__label__动力::1',\n",
       "   '__label__舒适性::-1',\n",
       "   '__label__操控::1']],\n",
       " array([[0.04436843, 0.04361726, 0.04059688, 0.03615582, 0.03563717]]))"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.predict([s_new], k=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf2 = fasttext.load_model(model_path+\".bin\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['__label__油耗::-1', '__label__舒适性::-1']]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf2.predict([\"顶起 垃圾 石墨 烯油 噪音 大 油耗 大 不到 5000 动力 衰减 的 就 很 明显 了\"], k=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
